from datasets import load_dataset, load_from_disk

# 加载磁盘数据（huggingface）
dataset = load_from_disk(r"../../dataset/mydata")
print(dataset)
"""
输出内容：
Dataset({
    features: ['id', 'conversations', 'category', 'subcategory', 'task'],
    num_rows: 1893
})
"""
#加载csv数据
dataset = load_dataset(path="csv",data_files="../../dataset/hermes-function-calling-v1.csv")
print(dataset)

# 加载磁盘数据（huggingface）
dataset = load_from_disk(r"D:\PycharmProjects\demo_14\dataset_test\dataset\ChnSentiCorp")
print(dataset)
# #取出训练集
# train_dataset = dataset["train"]
# print(train_dataset)
# #查看数据
# for data in train_dataset:
#     print(data)